package com.it.bigdata;

import com.it.bigdata.dao.TieZiDao;
import com.it.bigdata.domian.TieZi;
import com.it.bigdata.download.imp.BDTEDownLoader;
import com.it.bigdata.parse.imp.BDTBParsePage;
import com.it.bigdata.schedual.Schedual;
import com.it.bigdata.schedual.imp.BDTBSchedualForQueue;
import com.it.bigdata.schedual.imp.BDTBSchedualForRedis;
import com.it.bigdata.zookeeper.SpliderWatcher;
import com.it.bigdata.zookeeper.ZookeeperWatch;
import org.apache.commons.io.FileUtils;

import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class Splider {
    public static void main(String[] args) throws Exception {
        //创建下载对象与解析对象，并获取队列调度器
        final BDTEDownLoader bdteDownLoader=new BDTEDownLoader();
        final BDTBParsePage bdtbParsePage=new BDTBParsePage();
        //final BDTBSchedualForQueue schedual=BDTBSchedualForQueue.getBdtsq();
        //使用redis作为调度器来存储爬取的页面和当前页码
        final BDTBSchedualForRedis schedual = BDTBSchedualForRedis.getBdtbSR();
        final TieZiDao tieZiDao=new TieZiDao();
        //注册临时节点
        ZookeeperWatch.zkInit();

        //创建线程池
        ExecutorService executorService = Executors.newFixedThreadPool(5);
        //执行下载页面地址的任务
        executorService.execute(new Runnable() {
            public void run() {
                while (true){
                    long currentPageNum = schedual.getCurrentPageNum();
                    //下一页
                    schedual.nextPage();
                    //解析出具体的url并加入队列调度器中
                    String html=bdteDownLoader.downLoader("https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3&ie=utf-8&pn="+currentPageNum);
                    //解析出每一个贴子的地址
                    bdtbParsePage.parsePageList(html);
                }
            }
        });

        //此线程执行解析具体的帖子并获取其标题，图片和作者
        executorService.execute(new Runnable() {
            @Override
            public void run() {
               while (true){
                   //从队列中获取具体的地址
                   String pageUrl = schedual.pop();
                   if (pageUrl!=null){
                       //下载具体的页面然后进行解析
                        String html=bdteDownLoader.downLoader("https://tieba.baidu.com"+pageUrl);
                       TieZi tieZi = bdtbParsePage.parseCurrentTOtiezi(html);

                       try {
                           tieZiDao.add(tieZi);
                       } catch (IOException e) {
                           e.printStackTrace();
                       }
                       System.out.println(tieZi);
                   }else {

                   }
               }
            }
        });

    }
}
